# encoding: utf-8
"""
@author: 夏洛
@QQ: 1972386194
@file: 01-小说.py
"""

'''
使用异步下载小说

目录地址
https://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"4295122774"}

{title: "序章", price_status: "0", cid: "116326"} 第一章

详情地址  --- 第一章
https://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4295122774","cid":"4295122774|116334","need_bookinfo":1}
'''

import requests
import asyncio, aiohttp, aiofiles
import pymongo

from motor.motor_asyncio import AsyncIOMotorClient

client = AsyncIOMotorClient('localhost', 27017)
db = client['python']
collection = db['f']

headers = {
    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
}
b_id = '4295122774'
url = 'https://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"' + b_id + '"}'

import json, os, re


def validateTitle(title):
    # 去除字符串 特殊符号
    rstr = r"[\/\\\:\*\?\"\<\>\|]"  # '/\:*?"<>|'
    new_title = re.sub(rstr, "", title)
    return new_title


async def download(cid, b_id, title):
    data = {
        "book_id": b_id,
        "cid": f"{b_id}|{cid}",
        "need_bookinfo": 1
    }
    data = json.dumps(data)
    url = 'https://dushu.baidu.com/api/pc/getChapterContent?data={}'.format(data)
    async  with aiohttp.ClientSession(headers=headers) as session:
        async  with session.get(url) as resp:
            dic = await resp.json()
            # 异步写入文件
            # print(title)
            content = {
                'title': title,
                'content': dic['data']['novel']['content']
            }
            await save_data(content)

async def save_data(data):
    if data:
        return await collection.insert_one(data)


async def getCat(url):
    res = requests.get(url, headers=headers)
    dic = res.json()
    # 提取ID + 章节信息
    tasks = []
    for a in dic['data']['novel']['items']:
        title = a['title']
        cid = a['cid']
        # 准备异步任务
        tasks.append(download(cid, b_id, title))
    await asyncio.wait(tasks)


if __name__ == '__main__':
    # getCat(url)
    # asyncio.run(getCat(url))
    asyncio.get_event_loop().run_until_complete(getCat(url))